import requests
from lxml import etree
import re


url  = 'https://movie.douban.com/review/latest/'
headers = {
    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
    "accept-encoding": "gzip, deflate, zstd",
    "accept-language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7",
    "cookie": "bid=dKHr-SaabWU; _pk_id.100001.4cf6=c68680b6c3927ae3.1761703964.; __yadk_uid=wFDmK7cTLQpp7y2darZ8M90qXDBB7koA; ap_v=0,6.0; __utmc=30149280; __utmc=223695111; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1761976412%2C%22https%3A%2F%2Fwww.google.com.hk%2F%22%5D; _pk_ses.100001.4cf6=1; __utma=30149280.278698566.1761703964.1761974506.1761976413.3; __utmz=30149280.1761976413.3.3.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided); __utma=223695111.187055742.1761703964.1761974506.1761976413.3; __utmb=223695111.0.10.1761976413; __utmz=223695111.1761976413.3.3.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided); ll=\"108303\"; _vwo_uuid_v2=D8959A51A7C90FC859949F8FE747984ED|2193d555e47f3282edb285450172c3b3; __utmb=30149280.1.10.1761976413",
    "priority": "u=0, i",
    "sec-ch-ua": "\"Chromium\";v=\"142\", \"Google Chrome\";v=\"142\", \"Not_A Brand\";v=\"99\"",
    "sec-ch-ua-mobile": "?0",
    "sec-ch-ua-platform": "\"Windows\"",
    "sec-fetch-dest": "document",
    "sec-fetch-mode": "navigate",
    "sec-fetch-site": "none",
    "sec-fetch-user": "?1",
    "upgrade-insecure-requests": "1",
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36"
}
res = requests.get(url,headers= headers)

# print(res.status_code)
# print(res.text)
res_text = res.text
tree = etree.HTML(res_text)
s =tree.xpath('//div[@data-cid]')

pattern = r'<div id=".*?" class="review-short" data-rid="(.*?)">'
data_rids = re.findall(pattern, res_text)



for num,i in zip(data_rids,s):
    data_rid = num
    title = i.xpath('.//h2/a/text()')[0]
    content_start = i.xpath('.//div[@class="short-content"]/text()')[0]
    content_middle = re.sub(r'\s',"","".join(content_start))
    url_content = f'https://movie.douban.com/j/review/{data_rid}/full'
    res_content = requests.get(url_content,headers=headers)
    # print(res_content.json()['body'])
    page = etree.HTML(res_content.json()['body'])

    content_finally = page.xpath('.//div[@class="review-content clearfix"]//text()')
    text = ''.join(content_finally)  # 先拼成一大段

    text = re.sub(r'\s+', ' ', text).strip()  # 合并所有空白为一个空格
    print(f'电影名字：{title},电影评价：{text}')





# /div[@class='review-content clearfix']



# /div[@class='short-content']
#
#     <div data-cid="17176564">
#         <div class="main review-item" id="17176564">
#                 <a class="subject-img" href="https://movie.douban.com/subject/36596912/">                    <img alt="即兴谋杀" title="即兴谋杀" src="https://img1.doubanio.com/view/photo/s_ratio_poster/public/p2926251249.webp" rel="v:image" />                </a>
#
#
#
#     <header class="main-hd">
#         <a href="https://www.douban.com/people/200332916/" class="avator">
#             <img width="24" height="24" src="https://img2.doubanio.com/icon/u200332916-1.jpg">
#         </a>
#
#         <a href="https://www.douban.com/people/200332916/" class="name">Pig</a>
#
#         <span content="2025-10-31" class="main-meta">2025-10-31 20:04:21</span>
#
#
#
#
#     </header>
#
#
#             <div class="main-bd">
#
#                 <h2><a href="https://movie.douban.com/review/17176564/">锐评 即兴谋杀</a></h2>
#
#                 <div id="review_17176564_short" class="review-short" data-rid="17176564">
#                     <div class="short-content">
#
#                         刚从电影院出来，我真是草了。老子就想问问导演，你是不是心理有啥大病？你拍这片子纯纯就是为了折磨观众、喂观众吃屎是吧？ 首先，你TM管这叫“悬疑”？ 全片就搁那儿“强硬悬疑”，生怕别人不知道你“要开始悬疑了”。你那悬疑是悬疑吗？那叫故弄玄虚！有事没事就黑灯瞎火，...
#
#                         &nbsp;(<a href="javascript:;" id="toggle-17176564-copy" class="unfold" title="展开">展开</a>)
#                     </div>
#                 </div>
#
#                 <div id="review_17176564_full" class="hidden">
#                     <div id="review_17176564_full_content" class="full-content"></div>
#                 </div>
#
#                 <div class="action">
#                     <a href="javascript:;" class="action-btn up" data-rid="17176564" title="有用">
#                         <img src="https://img1.doubanio.com/f/zerkalo/536fd337139250b5fb3cf9e79cb65c6193f8b20b/pics/up.png" />
#                         <span id="r-useful_count-17176564">
#                                 32
#                         </span>
#                     </a>
#                     <a href="javascript:;" class="action-btn down" data-rid="17176564" title="没用">
#                         <img src="https://img1.doubanio.com/f/zerkalo/68849027911140623cf338c9845893c4566db851/pics/down.png" />
#                         <span id="r-useless_count-17176564">
#                                 2
#                         </span>
#                     </a>
#                     <a href="https://movie.douban.com/review/17176564/#comments" class="reply ">7回应</a>
#
#                     <a href="javascript:;;" class="fold hidden">收起</a>
#                 </div>
#             </div>
#         </div>
#     </div>




























# url  ='https://movie.douban.com/review/best/'

# headers = {
#     "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
#     "accept-encoding": "gzip, deflate, zstd",
#     "accept-language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7",
#     "cache-control": "no-cache",
#     "cookie": "bid=dKHr-SaabWU; _pk_id.100001.4cf6=c68680b6c3927ae3.1761703964.; __yadk_uid=wFDmK7cTLQpp7y2darZ8M90qXDBB7koA; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1761974505%2C%22https%3A%2F%2Fwww.google.com.hk%2F%22%5D; _pk_ses.100001.4cf6=1; ap_v=0,6.0; __utma=30149280.278698566.1761703964.1761703964.1761974506.2; __utmb=30149280.0.10.1761974506; __utmc=30149280; __utmz=30149280.1761974506.2.2.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided); __utma=223695111.187055742.1761703964.1761703964.1761974506.2; __utmb=223695111.0.10.1761974506; __utmc=223695111; __utmz=223695111.1761974506.2.2.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided)",
#     "pragma": "no-cache",
#     "priority": "u=0, i",
#     "referer": "https://www.google.com.hk/",
#     "sec-ch-ua": "\"Chromium\";v=\"142\", \"Google Chrome\";v=\"142\", \"Not_A Brand\";v=\"99\"",
#     "sec-ch-ua-mobile": "?0",
#     "sec-ch-ua-platform": "\"Windows\"",
#     "sec-fetch-dest": "document",
#     "sec-fetch-mode": "navigate",
#     "sec-fetch-site": "cross-site",
#     "sec-fetch-user": "?1",
#     "upgrade-insecure-requests": "1",
#     "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36"
# }
#
# res = requests.get(url, headers=headers)
#
# content_url ='https://movie.douban.com/j/review/17176124/full'
#
# headers2= {
#     "accept": "application/json, text/javascript, */*; q=0.01",
#     "accept-encoding": "gzip, deflate, zstd",
#     "accept-language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7",
#     "cache-control": "no-cache",
#     "cookie": "bid=dKHr-SaabWU; _pk_id.100001.4cf6=c68680b6c3927ae3.1761703964.; __yadk_uid=wFDmK7cTLQpp7y2darZ8M90qXDBB7koA; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1761974505%2C%22https%3A%2F%2Fwww.google.com.hk%2F%22%5D; _pk_ses.100001.4cf6=1; ap_v=0,6.0; __utma=30149280.278698566.1761703964.1761703964.1761974506.2; __utmb=30149280.0.10.1761974506; __utmc=30149280; __utmz=30149280.1761974506.2.2.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided); __utma=223695111.187055742.1761703964.1761703964.1761974506.2; __utmb=223695111.0.10.1761974506; __utmc=223695111; __utmz=223695111.1761974506.2.2.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided)",
#     "pragma": "no-cache",
#     "priority": "u=1, i",
#     "referer": "https://movie.douban.com/review/best/",
#     "sec-ch-ua": "\"Chromium\";v=\"142\", \"Google Chrome\";v=\"142\", \"Not_A Brand\";v=\"99\"",
#     "sec-ch-ua-mobile": "?0",
#     "sec-ch-ua-platform": "\"Windows\"",
#     "sec-fetch-dest": "empty",
#     "sec-fetch-mode": "cors",
#     "sec-fetch-site": "same-origin",
#     "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
#     "x-requested-with": "XMLHttpRequest"
# }
# res_content = requests.get(content_url, headers=headers2)
#
#
# # print(res.text)
# # print(res_content.json())
# res_html =res_content.json()['body']
# page = etree.HTML(res_html)
# content = page.xpath('//div[@id="link-report-17176124"]//text()')
# content = re.sub(r'\s',"","".join(content))
# print(content)